
# Notes ----------------------------------------------------------------------------------
#   Goal:   Download hourly AMPD data via FTP site (and unzip)
#   Time:   148 minutes


# Setup ----------------------------------------------------------------------------------
  # Load packages
  pacman::p_load(fastverse, here)
  fastverse_extend(topics = c('ST', 'DT'))


# Function: Download AMPD hourly data from FTP site --------------------------------------
  # Iterate over list
  download_ampd = function(i) {
    # Build the URL
    link_i = paste0(
      'https://gaftp.epa.gov/DMDnLoad/emissions/hourly/monthly/',
      combo_dt[i, year],
      '/',
      combo_dt[i, paste0(year, state, month, '.zip')]
    )
    # Download the file
    download.file(
      link_i,
      here('Data/electricity-generation/cems/raw/ampd-tmp.zip'),
      method = 'curl',
      extra = '--insecure'
    )
    # Unzip
    unzip(
      here('Data/electricity-generation/cems/raw/ampd-tmp.zip'),
      exdir = here(
        'Data/electricity-generation/cems/raw',
        combo_dt[i, year]
      )
    )
    # Delte zip file
    unlink(here('Data/electricity-generation/cems/raw/ampd-tmp.zip'))
    # Return something
    return('success')
  }


# Download the data ----------------------------------------------------------------------
  # Create all combinations of desired years, states, and months
  combo_dt = CJ(
    year = 2018:2022,
    state = state.abb %>% str_subset('AK|HI', negate = TRUE) %>% str_to_lower(),
    month = str_pad(1:12, 2, 'left', 0)
  )
  # Run the function
  blah = lapply(
    X = 1:combo_dt[,.N],
    FUN = function(x) tryCatch(download_ampd(x), error = function(e) NULL)
  ) %>% unlist()
  # Print the number of successes out of targeted links
  cat(paste0(sum(blah == 'success'), '/', combo_dt[,.N]), '\n')

